{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基础作业——皮马印第安人糖尿病数据集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 逻辑回归(LogisticRegression)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "#首先导入必要的模块\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "#读取数据\n",
    "data = pd.read_csv(\"diabetes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = np.array(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#数据标准化\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "#初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "#分别对训练和测试数据的特征进行标准化处理\n",
    "y_train = data['Outcome']\n",
    "X_train = data.drop([\"Outcome\"], axis = 1)\n",
    "X_train = ss_X.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### default Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "lr = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of each fold is: [0.49442383 0.50834194 0.4830274  0.43615578 0.48793522]\n",
      "cv logloss is: 0.4819768328671518\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cross_validation import cross_val_score\n",
    "loss = cross_val_score(lr, X_train, y_train, cv=5, scoring='neg_log_loss')\n",
    "print('logloss of each fold is:', -loss)\n",
    "print('cv logloss is:',-loss.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty = LogisticRegression()\n",
    "grid = GridSearchCV(lr_penalty, tuned_parameters, cv=5, scoring='neg_log_loss')\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.0008009 , 0.00080047, 0.00060077, 0.00080085, 0.00100102,\n",
       "        0.00060048, 0.00020018, 0.00312529, 0.        , 0.        ,\n",
       "        0.00332541, 0.00080109, 0.00100112, 0.00040603]),\n",
       " 'std_fit_time': array([4.00447874e-04, 4.00233904e-04, 4.90524413e-04, 4.00424071e-04,\n",
       "        1.16800773e-07, 4.90290765e-04, 4.00352478e-04, 6.25057220e-03,\n",
       "        0.00000000e+00, 0.00000000e+00, 6.16247930e-03, 4.00543412e-04,\n",
       "        2.13248060e-07, 4.97354871e-04]),\n",
       " 'mean_score_time': array([0.00040021, 0.00060039, 0.00080042, 0.00040045, 0.0004005 ,\n",
       "        0.00080085, 0.        , 0.        , 0.00312529, 0.        ,\n",
       "        0.        , 0.00060053, 0.00040021, 0.00020022]),\n",
       " 'std_score_time': array([0.00049015, 0.00049021, 0.00040021, 0.00049045, 0.0004905 ,\n",
       "        0.00040042, 0.        , 0.        , 0.00625057, 0.        ,\n",
       "        0.        , 0.00049033, 0.00049015, 0.00040045]),\n",
       " 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
       "                    100, 1000, 1000],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
       "                    'l2', 'l1', 'l2', 'l1', 'l2'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'split0_test_score': array([-0.69314718, -0.63247716, -0.64643499, -0.51842729, -0.49334858,\n",
       "        -0.4893871 , -0.49423847, -0.49442383, -0.49550595, -0.49553449,\n",
       "        -0.4956427 , -0.49565533, -0.49567367, -0.49566752]),\n",
       " 'split1_test_score': array([-0.69314718, -0.64153853, -0.65138433, -0.54263843, -0.51621248,\n",
       "        -0.50836483, -0.50930287, -0.50834194, -0.50874647, -0.50865995,\n",
       "        -0.50870199, -0.5086969 , -0.50870473, -0.50870065]),\n",
       " 'split2_test_score': array([-0.69314718, -0.63662203, -0.64570415, -0.52403018, -0.47822893,\n",
       "        -0.48374951, -0.48179749, -0.4830274 , -0.48313456, -0.48326216,\n",
       "        -0.48327803, -0.48329051, -0.48329118, -0.4832934 ]),\n",
       " 'split3_test_score': array([-0.69314718, -0.63189062, -0.64433664, -0.50307754, -0.45405914,\n",
       "        -0.44275292, -0.43661955, -0.43615578, -0.43568987, -0.43565059,\n",
       "        -0.43560726, -0.43560284, -0.43559749, -0.43559809]),\n",
       " 'split4_test_score': array([-0.69314718, -0.64067605, -0.64757466, -0.53287257, -0.49692637,\n",
       "        -0.48956528, -0.48810773, -0.48793522, -0.48809342, -0.48807935,\n",
       "        -0.48809911, -0.48809877, -0.48810315, -0.48810077]),\n",
       " 'mean_test_score': array([-0.69314718, -0.63664181, -0.6470899 , -0.52422544, -0.48778703,\n",
       "        -0.48280717, -0.48206439, -0.48202874, -0.48228703, -0.48229036,\n",
       "        -0.48231897, -0.48232204, -0.48232723, -0.48232527]),\n",
       " 'std_test_score': array([1.11022302e-16, 4.00318135e-03, 2.39292199e-03, 1.33688676e-02,\n",
       "        2.07318369e-02, 2.16460757e-02, 2.44379948e-02, 2.44134571e-02,\n",
       "        2.47961142e-02, 2.47956827e-02, 2.48335367e-02, 2.48355509e-02,\n",
       "        2.48414036e-02, 2.48395553e-02]),\n",
       " 'rank_test_score': array([14, 12, 13, 11, 10,  9,  2,  1,  3,  4,  5,  6,  8,  7]),\n",
       " 'split0_train_score': array([-0.69314718, -0.63735556, -0.64699594, -0.5192309 , -0.47577706,\n",
       "        -0.46925194, -0.46602435, -0.46594583, -0.46589352, -0.46589258,\n",
       "        -0.46589203, -0.46589201, -0.46589201, -0.46589201]),\n",
       " 'split1_train_score': array([-0.69314718, -0.63310132, -0.63649244, -0.51411049, -0.47224344,\n",
       "        -0.46704903, -0.46420481, -0.46410775, -0.46406259, -0.4640616 ,\n",
       "        -0.46406111, -0.46406111, -0.4640611 , -0.4640611 ]),\n",
       " 'split2_train_score': array([-0.69314718, -0.6356616 , -0.65056852, -0.51951737, -0.47886917,\n",
       "        -0.47222043, -0.46933007, -0.46925675, -0.46921157, -0.46921058,\n",
       "        -0.4692101 , -0.46921009, -0.46921008, -0.46921008]),\n",
       " 'split3_train_score': array([-0.69314718, -0.63804443, -0.65182756, -0.52743851, -0.48993887,\n",
       "        -0.48357524, -0.48097726, -0.48088442, -0.48084378, -0.48084284,\n",
       "        -0.48084241, -0.4808424 , -0.48084239, -0.48084239]),\n",
       " 'split4_train_score': array([-0.69314718, -0.63358836, -0.646171  , -0.51771935, -0.47759641,\n",
       "        -0.47164861, -0.46882413, -0.46875047, -0.46870587, -0.46870513,\n",
       "        -0.46870465, -0.46870464, -0.46870464, -0.46870464]),\n",
       " 'mean_train_score': array([-0.69314718, -0.63555025, -0.64641109, -0.51960333, -0.47888499,\n",
       "        -0.47274905, -0.46987212, -0.46978904, -0.46974347, -0.46974255,\n",
       "        -0.46974206, -0.46974205, -0.46974205, -0.46974205]),\n",
       " 'std_train_score': array([0.        , 0.00196666, 0.00539262, 0.00436479, 0.00596017,\n",
       "        0.00571694, 0.00585901, 0.0058569 , 0.00585954, 0.00585954,\n",
       "        0.00585957, 0.00585957, 0.00585957, 0.00585957])}"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4820287373783356\n",
      "{'C': 1, 'penalty': 'l2'}\n"
     ]
    }
   ],
   "source": [
    "#输出最好的模型\n",
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "C:\\Users\\YuGo\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZQAAAEKCAYAAAA1qaOTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8FPX5wPHPs7kJCZCEhECIAQS5Qjgih4AKoiBSBBG09aB4UP15oK3Wg3rXKrVWoFgRQUBFiYAo4gXiAWJRASFcIqecISHkJPfu9/fHbiBAAknYzWyS5/16rTuz852ZZyTZJ9+Z+T4jxhiUUkqp82WzOgCllFJ1gyYUpZRSbqEJRSmllFtoQlFKKeUWmlCUUkq5hSYUpZRSbqEJRSmllFtoQlFKKeUWmlCUUkq5ha8VOxWRMCAJiAP2AmOMMRkVtA0FtgGLjTH3uj77BogG8l3NrjLGpJ5rvxERESYuLu48o1dKqfpl3bp1R40xTc/VzpKEAjwKrDDGvCgij7rmH6mg7XPAt+V8fpMxZm1VdhoXF8fatVVaRSml6j0R+a0y7aw65XUtMNc1PRcYUV4jEekBRAHLaigupZRS1WRVQokyxhwGcL1Hnt5ARGzAy8DDFWxjtohsEJEnREQ8F6pSSqnK8NgpLxH5EmhWzqKJldzE/wGfGmP2l5MvbjLGHBSREGARcAvwVgVxjAfGA8TGxlZy10opparKYwnFGDOoomUickREoo0xh0UkGijvgnofoL+I/B/QEPAXkVxjzKPGmIOufeSIyLtATypIKMaYGcAMgMTERK3Vr5RSHmLVKa8lwFjX9Fjgo9MbGGNuMsbEGmPigIeAt4wxj4qIr4hEAIiIHzAM2FwzYSullKqIVQnlReBKEdkBXOmaR0QSRWTmOdYNAL4QkWRgA3AQeMOTwSqllDo3qU9PbExMTDR627BSSlWNiKwzxiSeq52OlFdKKeUWmlCUsliv2aPoNXuU1WG4hR6L96nJ49CEopRSyi00oSillHILTSiqVhr3+TjGfT7O6jCUUmVoQlFKKeUWmlCUUkq5hVXl65VSLk/N2+acqANn8PRYvE9NHof2UJRSSrmF9lBUrTT85WTnxBBr41BKnaQJRSkLGLsdR14ejrw8/EsM/r52Cn74CowBnOWQTpZFcr0bc2LyxDwAjtPmAYfjtHVL/2NObXv6O+bUadcyY8rMn9i/45RmYAj3KQIg77N3Kj74WqKuHEvpcTiy0rE1CvfovjShKHUOxhhMUZEzARw/juO46z2vgveKPsvNxnE8F0dePqao5MT2WwDgw56x91h1iG4Tih8Avz34vMWRnL+6ciylx1G8YyMBiQM9ui9NKKrOMQ4Hjrx81xe6KwHkVfBFf8p73sn2py2jpOTcOwaw2bAFBWAL8MXmL9hsJdhshfiRj823BFu4wdbMYAvww9YoHFuTSI5s3UKxw0bciJtcG3E9UE7ktGnO/LzMfs8gp68rzrXKzJ++/MSb2E75yPXhafNnrrv9zckAXHT7g2fGU8tsn/UKUPuPpfQ4/C7s4vF9aUJRtVJAkYPw7GL23nzzGUnB5OVVejsSEIAtOBhbgwYn3n1CQ/Fr1uyMz23Bwc6Xvw2bPQtbcTq2wlRs+QewHd+HLWcPUpx18jvW5gdhrSA8HsIvPPXVMPLEl/H/BnUEIP7O59z9v6nGHXnjNQB6jLnX4kjO35EZ/wVq/7GUHoetcYTH96UJRdU6WUuWcEFqAUZAxIZfZNSpX/gnphuUnxTKvItvBb8C9mLI3AfpO12vX+DoDtizC3IOndo2NAbC20Cb6yGirStptIFGseCjv2Kq/tCfdlVrOIqKOPLCC2S+N59Cf0gLh8Fvl/vk58oxBrIPl0kaOyF9F6TvgIy94ChzmiuwsTNZtL7MmSzCXYkjrDX4NzjvY1OqLtCEomqF4kOHOPDAgxQkJxN2222s++LNU68rnE1BVplkcVryKMo92c43EMLaQGRH6DDcmTBKexwNwjxzYMCcMf4ADPXYHmrOMzd1APRYvElNHocmFOX1clev5tBfHsIUF9Ni6hRCr7oKls0+tVFJIRzbU05vYyccTz3ZTmzQONaZJGL7nDw9Fd4WQluUf3Hbw37za1Pj+1TKEzShKK9lHA7SX3+dtKn/IeDCC2kxdQoBrVoBYLMZLozJgXdGOZNG5r6T4yIAgiOdyaLd4FMvhoe1At8Ai45IqbpNE4rySvasLA799RFyv/2W0N/9juhnnsbW4OS1igNXFzIkKwdyj0CLHtDlhpO9jbA2ENTYwuiVqp80oSivk79lCwfvn0BxaipRTz5Bk9//Hjntekn/vFz2+foR+6dVlb+WopTyKE0oyqtkLlxIyrPP4RMWRtw7bxOUkHBmo/RdtC0u5J9NIuh3aDUO46DEUYLDOLAbO3aHHbuxn5w3dhyOMtMVtXOcus7p8+WuV86+Tl+v2G6nqKSEIrvzVeKwn3jZjZ1iRzEA8W/2dg09LDtwseygxLKfCScGKp72uVTY5sz2cuLzCrYhJ6dPb3si1jJtcgoLAeg35+YK/41P7qvchdVb76zLqrc0t6AYEC6fW7vLDTuPAzan7KNzs1iP7ksTivIKjoICUp57jqxFHxB8SR+a/+tf+IaVf2fVoZ9n87dmkfwUFMi8L+92y/5tYsMmNnzE58TLZjs5f2KZzTkv2DDGhsMIDgfYHYLdLpQ4hGK7c2B9ketljA1jAgEbGAFs2MSHID9fGvj6klGYAeKgVaMWrmpbjhO1s4zrE2NMaYUv52fGnFhaWqPLYE5bz7ktyv385HbLznP6No1xft8ag6N0W5y6rdL6YwYQWwEA+Y5jVf43OLG9aqx51qXmLMvPlml8nTWwskryqxGTF/F1JpTjRYWe35XH96DUORTt38+BCRMo3LqN8Lv+RNP77kN8fM5oZ4zhw50fMunAR9gD/IkqFl6+9u1Tv/BPSwQnltnKTwyl8yKCw2HIyCsiLbeQ1OxC0nIKSct1vqfmFJKWU+D8LKeQ7IIzS7GIQHiwPxENA4gMDaRpkwCahpx8RZaZDgnwPXEar9fsUQB8ctNkz/6PrgGlx/LDuEUWR3L+6sqxlB5Hr9i2Ht+XJhRlqZxvvuHQXx8BIOa1/xIyYEC57Y7mH+WZ/z3DN/u/oUdBAVElYXwTHENC03JOiZ0mr6jkRCJIy8lzJYeTCSPVlSiO5hZhd5z512wDf58TyeCiZiH0uzDCmTAanposwoL98fXRRwyp+ksTirKEsdtJmzaN9NemE9ChAzFTp+DfsmW5bZf/tpzn/vccx4uP81DwRdyyfxWDojviKAlg88GsE72IM165haRmF3C8yH7GNm0CEQ1PJoOO0aHO5FDau3BNNw0JIDhAf02Uqgz9TVE1riQjg0N/eYjj339Po1HX0eyJJ7AFBp7RLqswixd+fIFPdn9Cx/CO/KPPM7SZeTVcNJSMtAAy9g1j2H++O2WdkEDfE8mgU/NQBlwUWe5ppyYN/PGx6d1hSrmTJhRVo/I3buTAAw9iT08n+u/P0fj668tt9/3B73ni+ydIz0/n7oS7ubPLnfjt+BLyj0GXG8hf9AtiK+a1P/SmaUggkSEBRDQMIMj/zGsvSqmaoQlF1QhjDBnvvceRF17ELzKSC957l6BOnc5ol1ecx7/X/Zuk7Um0btSaqQOm0inC1S45CRqEk9vyMgpyCggK3cWQziNr+EiUUhXRhKI8zpGXx+GnnyZ7yccEX3YpLSZNwqfxmSPZN6RuYOJ3E9mfs59bOt7C/d3uJ9DXdSqsIAt++RR6jOWTLWlg/AhstKOGj0QpdTaaUJRHFe7Zw8H7J1C4cycR999HxF13IacVYCyyF/HqhleZs2UO0cHRzBo8i4ubXXzqhrYuAXshdLmRBR8fwMc/E7/AtBo8EqXUuWhCUR6TvXw5hx97HPH1peUbb9CwX98z2mw/tp3HvnuMHRk7GNV2FA9f/DDBfsFnbiw5CcLasNu/HWt/W0nDiB1acUUpL6MJRbmdKSkh9ZVXODbrTQLj44mZMhm/5s1PaVPiKGHOljm8uuFVGvk3YtrAaVzW8rLyN5i5H/augssfZ+H6g/jYhMBGO2vgSJRSVWFJQhGRMCAJiAP2AmOMMRnltLMDm1yz+4wxw12ftwLmA2HAeuAWY0yR5yNX51KSlsbBP/+FvJ9+ovHvbyTqscew+fuf0ua37N94/LvHSU5L5qoLruJvvf9Gk8AmFW9080IA7PFjWPT6Li5r15QtppaXw1CqDrJqWO+jwApjTFtghWu+PPnGmK6u1/Ayn08CXnGtnwHc7tlwVWXkrVvHnutGkb9pE80nvUj0U0+dkkwcxsG7297l+iXXszdrL5P6T+Jfl/3r7MnEGNiYBC17sfJoMEeyCxmTGFMDR6OUqiqrEsq1wFzX9FxgRGVXFGcBpIHAwuqsr9zPGEP6nDn8dutYpEEQcUlJNLr22lPapBxP4U/L/8QLP75Aj2Y9WHztYoa2HnpGWfozpGyCtG3Q5QYWrj1AWLA/A9tHefBolFLVZdU1lChjzGEAY8xhEYmsoF2giKwFSoAXjTEfAuFApjGmtDrfAaCFxyNW5bLnHufw3/5Gzuef03DQFTR/4QV8QkJOLDfGsHT3Ul744QVKTAlP9H6C0e1GnzuRlEpOApsfWa2vYfmHP3NT71j8fbVellLeyGMJRUS+BJqVs2hiFTYTa4w5JCKtga9EZBOQXU67CutTi8h4YDxAbKxnnwVQ3xTu3MmB+ydQtHcvkQ/9hbDbbz8lURwrOMaz/3uWFftW0C2yG8/3fZ6WoeXX6yqXww6bFkDbq/hwewFFdgeje1RhfaVUjfJYQjHGDKpomYgcEZFoV+8kGkitYBuHXO+7ReQboBuwCGgsIr6uXkoMcOgsccwAZgAkJiZW94EL6jRZn3zC4SeexBYUROzs2QT36nnK8hX7VvDs/54lpyiHP/f4M7d2vBUfWxXLouz+xvmI34QbWLBiP52ah9Kxeaj7DkIp5VZWnTtYAox1TY8FPjq9gYg0EZEA13QE0BfYapxPy/kauP5s6yvPMEVFpDz/Dw795SEC27en1QcfnJJMcopymPjdRB74+gGiGkSRNCyJcZ3HVT2ZACS/DwGN2NawD5sPZjMmUXsnSnkzq66hvAi8LyK3A/uA0QAikgjcZYy5A+gAvC4iDpyJ70VjzFbX+o8A80Xk78DPwKyaPoD6qPjIEQ4+8CD5P/9M2NhbiXzoIcTP78TyNYfX8MTqJ0jLS2N8l/Hc1eUu/Hz8zrLFsyg6Dts+hvjreX9jGv4+Nq7t2vzc6ymlLGNJQjHGpANXlPP5WuAO1/T3QHwF6+8Gepa3THnG8TVrOPjnv+AoKKDFK/8m9OqrTyzLL8ln8rrJvPvLu8SFxvH21W8T37Tcf7rK++UTKD5OcafRfPTuIa7sGEXjBidvQY4reuj8tq+UcjsdKa/OyjgcpM+cRdrkyfi3asUFU6cQ0KbNieXJaclM/G4ie7P3clOHm5jQfQJBvkHnv+PkJGgUy4q81hw7voHr6/DYk9r+iFmlSmlCURWyZ2dz6NHHyP3qK0KHXk30c89hC3bW2Sq2F/PaxteYtXkWUQ2imHnVTHpF93LPjnOOwK6voN+DLFh3iKjQAC5t29Q921YeVZeSY105lpo8Dk0oqlwF27dz4L77KT50iKjHH6fJLTefuCX414xfmfjdRH459gvXtrmWR3o+Qoh/yDm2WAWbF4FxkN56BF9/uZ8/XdbmjKcrJv2pj/v2p5RyC00o6gyZH35IytPP4BMaygVvzaVB9+4A2B125m6dy7SfpxHiH8KUAVMYGDvQ/QEkJ0F0VxbsC8ZhYHSPunu6S6m6RBNKPTPu83EAzB4y+4xljqIijjz/DzKTkmjQsyct/v0yvhERAOzP3s/E1RP5OfVnBsUO4ok+TxAWGOb+ANO2w+ENmMH/YMH3+0m8oAmtmzZ0/36UUm6nCUUBUHzwIAcmPEDB5s2E33kHTSdMQHx9Mcaw4NcF/Gvtv/AVX/7R7x8Maz2s8qVTqio5CcSHTWFXsittB5NGtfbMfpRSbqcJRZG76jsOPfQQxm4nZtp/CBnkLHJw5PgRnvr+KVYfWk2f6D482/dZmgWXV03HTRwO52DGNgN4b0shQX4+XNNFx54oVVtoQqnHjMPB0f++xtFXXyWgXTtipkzGPy4OYwyf7vmU5394nhJHCRN7TeSGi27wXK+k1L7/QdZ+ii77G0s/OsTV8c1oGKA/okrVFvrbWs/c+J8tAJT0yuDQXx/h+KpVNLp2OM2efhpbUBAZBRk8t+Y5lv+2nISmCTzf73kuCL2gZoJLTgK/YD63dyencIcWglSqltGEUg/5FdrZO+p6StLSaPb00zS+YQwiwrf7v+Wp758iqyiLCd0nMK5TNWtwVUdxAWz5EDr8jvkbjhEb1oBerTxw0V8p5TGaUOqZ4OxiwtIKMM0bccG78wiKjye3KJd//vRPFu9cTLsm7Xj9yte5KOyimg1sxxdQmEVq6xF8/2M6f76yHTabh0+xKaXcShNKPVK4YwdhaQUUBPnQbtEifJs04aeUn/jbd38jJS+FO+Lv4O6Eu/H38T/3xtxtYxI0jOK9tDhEdjNKx54oVetoQqlH0qZOxdggPSqIkpAgXv5xEu9se4fYkFjmDplL18iu1gSWdwx2LMP0HM+C9Yfp2yaCFo3dUA9MKVWjNKHUE/mbNpGz/Euym/jzW4ThP0vHsCdrDzdedCMP9niQBn4NrAtuywfgKGZj2BAOZOTw8OAaPt2mlHILTSj1RNrkKfg0bszSi/NZ2q2QiOLjvH7l61zS/BKrQ3OOPWnagbm7QwgJzGdwJw+OdVFKeYxVT2xUNej4jz9yfPVqCv4wlCU97DQKasLiaxd7RzI5tgf2/0BBp9F8tiWF4QnNCfSroTvLlFJupQmljjPGkDZ5Cr6RkbwS+wt+Nj/iGsUR6u8lz2ZPfh8QvpD+FBQ7GK2P+VWq1tKEUscdX7mS/PXrSb9xIGuzkmke3Bwf8ZIegDHOwYxx/Zi7pZi2kQ1JiGlkdVRKqWrShFKHGYeD1ClT8GsZw0uR64kLjSMiKMLqsE46uA6O7eJIq2tZvy+T0Ykxni/vopTyGE0odVjOsmUUbt3Gb9f34dfju5nQfYJ3fWEnJ4FvIPOyu+JjE0Z0a2F1REqp86AJpY4yJSWkTf0Pfm1a80Lj7+gS0YUrYq+wOqyT7MWweRGOdlfzXnIWAy6KJDIk0OqolFLnQRNKHZW15GOKdu9m88h4jhSk8UCPB7yrd7JzBeSlsyl8MGk5hYxO1JHxStV2mlDqIEdREUenTcOvY3v+2eBb+rfoz8XNLrY6rFMlJ0FQGDMOtSY82J+B7SOtjkgpdZ40odRBmQsWUHzoEN//Lo7c4uNM6D7B6pBOVZAF2z+loP0Ilv2SzohuLfDz0R9FpWo7HSlfxzjy8zk6fTq+3bsw1Xclw+KG1Xzl4HPZ9jGUFLDc93KK7UZPdylVR+ifhXVMxrx52NOO8vlV4Tgw3NPtHqtDOlNyEoS15r87mtAlphHtm3nJIEul1HnRHkodYs/JIf2NmUifHsy0ream9jfRouGpt+LOHjLbouhcsg7CnlUc6f4A277P4blrO1kbj1LKbTSh1CHHZs/BnpXFgst8aeDbgDvj77Q6pDNtWgAY5hf2wd/XwfAEHXuiVF2hp7zqiJKMDI7NmYP98p4skHWM6zyOJoFNrA7rVK5SK46Yi5m9TbiqYxSNGvhZHZVSyk00odQR6TPewFFQwJu984kIiuDmDjdbHdKZjmyG1K1siRhKZl6xFoJUqo7RhFIHFB85Qsa775J/RU+WyzbuTrjb2gdmVWTjfLD5Mv1oF6IbBdLvQi+qK6aUOm+aUOqAo6+9hrHb+U+3I8SFxjGy7UirQzqTww6bFlIQdwWf7SpkVPcYfGxeNHJfKXXeNKHUckX795O5cBEZV/Vgrc9+7ut2H342L7wusWcl5KbwTeBAHAau76FjT5SqayxJKCISJiLLRWSH673cq8ciYheRDa7XkjKfzxGRPWWWda256L3L0WnTEB8f/tXpN+Ij4rnygiutDql8yUmYgFBe+a01PePCiIsItjoipZSbWdVDeRRYYYxpC6xwzZcn3xjT1fUaftqyh8ss2+DRaL1U4Y4dZC35mINDEtjhe5QHezzoXQUgSxUdh20fczT2aranF3O9joxXqk6yKqFcC8x1Tc8FRlgUR62WNvU/SIMgJrX7lb4t+npfAchSv3wKRbksKulLA38fromPtjoipZQHWJVQoowxhwFc7xWVmg0UkbUiskZETk86z4tIsoi8IiIBHo3WC+Vv2kzO8uXsGNyBFN/jPNj9QatDqlhyEo7QGKbtasrQ+GiCA3Q8rVJ1kcd+s0XkS6BZOYsmVmEzscaYQyLSGvhKRDYZY3YBjwEpgD8wA3gEeLaCOMYD4wFiY2OrsGvvljZlCtIolElttnFN62u8rwBkqdxU2PUVv7a5jdxUwxgde6JUneWxhGKMGVTRMhE5IiLRxpjDIhINpFawjUOu990i8g3QDdhV2rsBCkVkNvDQWeKYgTPpkJiYaKp1MF4m76efOP7dd2wYk0Ce/6/c2+1eq0Oq2OZFYOy8kZVIXHgDLo7zstH7Sim3seqU1xJgrGt6LPDR6Q1EpEnpqSwRiQD6Altd89Gud8F5/WVzDcTsFYwxpE6eAk3DefmCrdxw0Q1nFID0KslJFDWNZ9H+EK7vEeOdNw0opdzCqoTyInCliOwArnTNIyKJIjLT1aYDsFZENgJfAy8aY7a6ls0TkU3AJiAC+HuNRm+h46tWkb9uHauuiMQ3qAF3dvHCApCl0n6FQz/zffAViMB13fXuLqXqMkuujhpj0oEryvl8LXCHa/p7IL6C9Qd6NEAvZRwOUidPxkRH8t/YX7m7032EBYZZHVbFkpMwYuPlw13od2EEzRsHWR2RUsqDdKR8LZKzbDmFW7fxyYCGNA6O4JaOt1gdUsUcDkh+n8xmfdmUFagX45WqB6qcUETEJiL6iL0aZux20qZOpSQ2mrdb/ua9BSBL7V8DWftYSn9CA325smOU1REppTysUglFRN4VkVARCcZ5YXy7iDzs2dBUWVlLPqZo926SLhVaNrqA69pdZ3VIZ5echPEL5uX97bi2awsC/Xysjkgp5WGV7aF0NMZk47yj6lMgFvDi8y11iykq4ui0aRRc2IKPYo5wf/f7vbMAZKniAtiymD1NB5BZ4s9oLbWiVL1Q2YTiJyJ+OBPKR8aYYqBOjOmoDTIWLqT44EFmX1JIp4jOXHXBVVaHdHY7lkFBFnOP9+aiqBDiWzSyOiKlVA2obEJ5HdgLBAMrReQCINtTQamTHPn5pL82newOMXzdPMN7C0CWlZxESYNI3jlyAaMTdeyJUvVFpRKKMWaqMaaFMWaocfoNGODh2BSQ8e67lKSlMb13Nn1b9KNXdC+rQzq7vGPw6xesCxmI2HwZ0c2LB10qpdyqshflJ7guyouIzBKR9UC9HAtSk+y5uaTPeIOjCS1Z2yyPB3o8YHVI57b1Q3AUM/VoDwa2jySiYb2r26lUvVXZU163uS7KXwU0BcbhGt2uPOfY7DnYs7KYmniUoa2G0j6svdUhndvGJHJDL2T18eaM1rEnStUrlU0opSfBhwKzjTEby3ymPKAkI4Njc+awPzGGnc2MdxeALHVsD+xfwzLfy4loGMDlFzW1OiKlVA2qbEJZJyLLcCaUL0QkBHB4LiyV/sZMHPn5TOl2hBsuuoGWIbXgr/1NCwB4JSWBkd1a4OejhRiUqk8qW8vrdqArsNsYkyci4ThPeykPKD6SSsa8efx6cTOONsvlzngvLgBZyhhITuJw4x7sTwnX011K1UOVSijGGIeIxAB/cN0C+q0x5mOPRlaPHZ3+Gg57CVO6H+GPne8hPCjc6pDO7dB6SN/J/KD7SGjZmHZRIVZHpJSqYZW9y+tFYALOsitbgftF5AVPBlZfFe3fT+aChfzcKxx7s3DGdhx77pW8wcYkHD4BzM7owugeOjJeqfqosqe8hgJdjTEOABGZC/yM81G8yo2OTnsVYxNe75bO/yX8zbsLQJayF8PmRWwN6UthYQi/S2hudURKKQtU5app4zLTWkvDAwp37iRryRK+6x1Cw+axjGo3yuqQKmfXV5B3lBmZiQzu1IxGQV5cZ0wp5TGV7aG8APwsIl/jvF34UrR34nZpU/+DI9CfOd2yeLLbE95dALKs5CSK/BvzWXZn3tRCkErVW5W9KP+eiHwDXIwzoTxijEnxZGD1Tf7mLeQsW8YXA0Jp2fIirorz8gKQpQqy4ZdPWBl4JZGNQ7ikTYTVESmlLHLWhCIi3U/76IDrvbmINDfGrPdMWPVP2pQplDQMZH6340zp/gA2qSVjOLZ9DCUF/PdYD0Zd3gIfm453Vaq+OlcP5eWzLDNoPS+3yFu7luOrVvHhlQ3oGncJfZr3sTqkyktOIjMwhvUFbZncQ8eeKFWfnTWhGGO0orCHGWNIfWUyhY0b8GFCIW/XhgKQpbIOYvas5EO/MfRqFU5seC24I00p5TGVuoYiIuU9bzYL2GSMSXVvSPXL8e++I3/dOuYPCeCKdkPpGN7R6pAqb/NCBMPs3F7cN0R7J0rVd1UpvdIH+No1fzmwBmgnIs8aY972QGx1njGGtFcmczwimC8TSljc9T6rQ6qajUn8FtSJo44WDI1vZnU0SimLVfbKrwPoYIwZZYwZBXQECoFewCOeCq6uy1m2nIKtW3mrdyHXdRxDy9Ba9Fd+ymZI3cJbx3sxrEtzGvhX9m8TpVRdVdlvgThjzJEy86lAO2PMMREp9kBcdZ6x20mbOpWMZsH8mGDjky5/sjqkqklOwiG+fFDUkzd07IlSisonlFUishRY4Jq/Huez5YOBTI9EVsdlffwxRbt28eZIG2Pj760dBSBLOeywaQHr/XvQJDiaHhc0sToipZQXqGxCuQe4DuiHc2DjXGCRMcagz5avMlNUxNFpr5ISE8zOLg2Y0ulWq0Oqmr2rIOcwbxaNZtSVMbgqUCul6rnKjpQ3IvIdUIRz/MmPrmSiqiFz0SKKDxxg1hikigTkAAAdN0lEQVQb47v+mWC/YKtDqpqNSRT6BPO16c6T3fV0l1LKqbLl68cAP+I81TUG+EFErvdkYHWVo6CAtP++xt64INK7tGR0u9FWh1Q1RXmYbUv4wvSmV7sWNGsUaHVESikvUdlTXhOBi0vHnIhIU+BLYKGnAqurMua9iz0tjTcH+3Bf9/vx86klBSBLbf8UKcrl3aI+3KIj45VSZVQ2odhOG8CYTtVK3yvAnpvL0Rkz2NY2EEloy5BWQ6wOqeqSkzjmG8l2iWdQx0iro1FKeZHKJpTPReQL4D3X/A3Ap54Jqe46Nmcujqws5oz04dEetagAZKncNMzOFSywD2N4jxgCfH2sjkgp5UUqe1H+YREZBfTFeZfXDGPMYo9GVseUZGSQPns26zv4E9W9J5c0v8TqkKpu8yLE2FlY3JdXEvV0l1LqVJUe3myMWQQscsdORSQMSALigL3AGGNMRjntYoGZQEucd5cNNcbsFZFWwHwgDFgP3GKMKXJHbJ6SPnMmjrzjvN3Ph5drUwHIspKT2OXTBp+oDnRqHmp1NEopL3PWcy4ikiMi2eW8ckQk+zz2+yiwwhjTFljhmi/PW8BLxpgOQE+cI/QBJgGvuNbPwFlrzGsVH0nl2Dvv8H1nX+ITr6ZTeCerQ6q6ozvg0HreLejNmMSWOvZEKXWGsyYUY0yIMSa0nFeIMeZ8/kS9FufgSFzvI05vICIdAV9jzHJXLLnGmDxxfpMN5OQdZuWu703SX5+Oo6SYBf1s3NetlhWALJWchAMbn9GXEd1aWB2NUsoLWXVVOMoYcxjA9V7e7ULtgEwR+UBEfhaRl0TEBwgHMo0xJa52BwCv/YYrOnCAjKT3WdFF6N9rNLGhsVaHVHXGYJKT+IF4urRvT1iwv9URKaW8kMdKxIrIl0B5Nc0nVnITvkB/oBuwD+c1lz8CS8ppW+GofREZD4wHiI2t+S/zo9NepcRm+OTSIOYl3FXj+3eLfWuQzH28X3QNo7UQpFKqAh5LKMaYQRUtE5EjIhJtjDksItGcvDZS1gHgZ2PMbtc6HwK9gTeBxiLi6+qlxACHzhLHDGAGQGJiYo2WiynctYvMJR/xWSIM7zOOiKCImty9+yQnUSiBrGvQl5faNbU6GqWUl7LqlNcSYKxreizwUTltfgKauEblg/O6yVZXDbGvcZaBOdv6lkubOpViP+Gby8MZ23HsuVfwRiWFOLYs5rOSHlzd40J8fWrZ2BmlVI2x6tvhReBKEdkBXOmaR0QSRWQmgDHGDjwErBCRTTjHv7zhWv8R4M8ishPnNZVZNRz/OeVv2ULOF8v4KNFwU5+7aOjf0OqQqmfHMmwFmSy292O0llpRSp2FJY/ZM8akA1eU8/la4I4y88uBLuW0243zNmKvlTZ5CnkNfFg/sDmP1rYCkGWY5CQypDG5LfpxYWQtTYpKqRqh5y88IG/dOo6vWsUHvQx39pmAv08tvSsqPwOz/QsWF/dhVGKc1dEopbycJhQ3M8Zw5N//JruhD3sGtefqVldbHVL1bfkQm6OIT+RShiVEWx2NUsrLaUJxs+PfraZg3XoWXGK4p/efa18ByDIcG+ezixbEduxNaGAtK7OvlKpxtffbzgsZY0h55d8cbWwje3Cv2lkAslTGXmz717CouB9jLq6FgzGVUjVOE4ob5SxfTvHWbST1hft7/bl217vatACAHxoOpHfrcIuDUUrVBpbc5VUXGbudlMmvcCjcRuA1g+kc0dnqkKrPGIp/ns96R3v6JXbHZqvFiVEpVWO0h+Im2UuXYt+9l6RLbdyXOMHqcM7PoZ/xy9jJB/b+XN9DS60opSpHeyhuYIqKODx1CnuihNjfjeGC0AusDum8mI3zKcKPoy2H0DKsgdXhKKVqCe2huEHmBx9gDh7mg4GB3NXtbqvDOT/2YkqSF/KlvRvDerW3OhqlVC2iCeU8OQoKODztP/wSAwm/+2PtLQBZatfX+BWk84XtMoZ00rEnSqnK04RynjLefQ85eoyPBzViXOfbrA7nvBVvmE+maUho/NUE+ftYHY5SqhbRhHIe7Lm5pEz/LxtaCQOH31t7C0CWKsxBfvmEj+29GXlxa6ujUUrVMppQzkP6nDnYsnP5akgUYy4aY3U452/bx/g6CvgxdBDdYxtbHY1SqpbRu7yqqSQjg7RZM/mpnTB82J9rbwHIMvLWvkuaI5KOF19ZuwdlKqUsoT2Uakp74w0oKOTHYa25pvU1Vodz/rIPEXTgOz4y/RilY0+UUtWgPZRqKE5N5dg7b/NdR+EP1zxaqwtAlnIkL8SG4VDscCJDA60ORym3Ky4u5sCBAxQUFFgditcKDAwkJiYGP7/qFYPVhFINKf+dhikpYft13RjfvK/V4bjF8bXz2Om4kMt697Y6FKU84sCBA4SEhBAXF6endMthjCE9PZ0DBw7QqlWram2j9v9pXcOKDhwke8EivkoQxl31WN34wTyyhZDMX/jC51Ku6BBldTRKeURBQQHh4eF143fWA0SE8PDw8+rBaUKpogNTX8aOg9QxlxPfNN7qcNyiYN17FBsfbJ1H4e+rPxKq7qpqMrnh9f9xw+v/81A03ud8k61+e1RB4a5dFHz8Oct6+HDHwL9aHY57OOzYNybxraML1/SpGwlSKW/VsOHJsWpDhgyhcePGDBs2rNy299xzD127dqVjx44EBQXRtWtXunbtysKFC6u0z/Xr1/P555+fV9yVpddQqmDfv/9JgZ+h5KbhxDWKszoc99j7HcGFqfwUOo7HmjeyOhql6o2HH36YvLw8Xn/99XKXv/rqqwDs3buXYcOGsWHDhmrtZ/369WzevJkhQ4ZUO9bK0h5KJRVs3UrJipV80cuf2/s/aHU4bpOx5h1yTBAxvUZaHYpS9coVV1xBSEhItdbdsWMHgwcPpkePHlx66aX8+uuvAMyfP5/OnTuTkJDAgAEDyM/P59lnn2XevHnV6t1UlfZQKmHc5+MYMX0LzQOh0dibadqgqdUhuUdRHg12LuVj04thPdpYHY1SNeaZj7ew9VD2OdttPexsU5nrKB2bh/LU7zqdd2yVMX78eGbOnEmbNm1YvXo19957L8uWLeOZZ57hm2++ISoqiszMTIKCgnjyySfZvHkzkydP9nhcmlAq4dZ/bqTZoXwWDWrAhJ53WR2O25Rs+5QARx4HWv6OJsG1f6S/UvVBZmYma9asYdSoUSc+KykpAaBv377ceuutjB49muuuu67GY9OEUgn+uYVkBEPcbf9HiH/1uqje6Nj/3qLEhJHQr/yLgkrVVZXtSZT2TJL+1MeT4VSJMYaIiIhyr6m88cYb/PDDDyxdupSEhASSk5NrNDa9hlIJcwbZeOcKH8Yk3GJ1KO5z/CjhKatY7nMp/dtFWh2NUqqSmjRpQnR0NIsXLwbA4XCwceNGAHbv3k3v3r157rnnaNKkCQcPHiQkJIScnJwaiU0TSiXk9e1Car/2daIAZKmctfPxwUFJ5zH4+uiPgVI1rX///owePZoVK1YQExPDF198Uel158+fz/Tp00lISKBTp04sXboUgAcffJD4+Hji4+MZNGgQnTt3ZuDAgWzcuJFu3brpRXlv4O/jX6eSCUDe2vfY77iAAf0vszoUpeqN3NzcE9OrVq2q1DpxcXFs3rz5lM9at25dbgJasmTJGZ81bdqUtWvXVjHS6tGEUg+ZozuIytnMZ6F38MemtfyhYEp5kDddO6kNNKFUwuwhs60OwX1mX0NK2lEijdC45++tjkYpVYfoyfP6xhgC8lJYQ2eu6JlgdTRKqTpEE0o9U1iQRxiZ/NZiGCGB1XvmgVJKlUcTSj3za2oOeSaANpf+wepQlFJ1jCUJRUTCRGS5iOxwvTepoF2siCwTkW0islVE4lyfzxGRPSKywfXqWpPx11Zm19e0MftY5Yjn4nYtrQ5HKe83+xrnS1WKVT2UR4EVxpi2wArXfHneAl4yxnQAegKpZZY9bIzp6npVrwxnfeFwULBiEubtkew3kWy3tcZm04cMKVXTarp8/eLFi3nppZfOO+7Ksuour2uBy13Tc4FvgEfKNhCRjoCvMWY5gDEmF1V1+Rlkv3sbofu/4iN7X3bYWjHEf6PVUSlV77mrfH1JSQm+vuV/lY8cWbNVxK3qoUQZYw4DuN7Lq/3RDsgUkQ9E5GcReUlEfMosf15EkkXkFREJqImgaxtz6Gdyp15C4L5v+ZfvnbS84x0eittLZ33uiVKWO5/y9f369WPixIlceumlTJs2jY8++ohevXrRrVs3rrrqKlJTnSdzZs6cyQMPPADAzTffzIQJE7jkkkto3br1idIt7uSxHoqIfAk0K2fRxEpuwhfoD3QD9gFJwB+BWcBjQArgD8zA2bt5toI4xgPjAWJjYysdf61mDAU/zsHn84fJdoQwM/oV7rvl91pRWKlSnz0KKZvO3S7FVVyxMtdRmsXD1S+eX1xVkJ2dzcqVKwHIyMhg+PDhiAjTp0/n5ZdfZtKkSWesk5qayurVq9m0aRNjxoxxew/GYwnFGDOoomUickREoo0xh0UkmlOvjZQ6APxsjNntWudDoDcwq7R3AxSKyGzgobPEMQNn0iExMdFU72hqkeJ8shZOoNH2JFY54vm17yv87crEk9dMxn1ibXxKKbe48cYbT0zv27ePMWPGkJKSQmFhIe3atSt3nREjRiAidOnShYMHD7o9JquuoSwBxgIvut4/KqfNT0ATEWlqjEkDBgJrAcokIwFGAJvLWb/+ObabzDk30jh7OzNkNJ1veZ7b20ZZHZVS3qeyPYnSnokX/iEWHBx8Yvqee+7h8ccfZ+jQoXz55Ze8+GL5xxcQcPLqgDHu//vaqoTyIvC+iNyO83TWaAARSQTuMsbcYYyxi8hDwApX4lgHvOFaf56INAUE2ADUnadeVVPR5o+xf3AX2A0vhD3L7eP+RGRooNVhKaVqQFZWFi1atMAYw9y5cy2Lw5KEYoxJB64o5/O1wB1l5pcDXcppN9CjAdYm9hIyP3mKxuunkexoxffd/83Dv7tcS9Ir5eX69+/PL7/8Qm5uLjExMcyaNYvBgwdXa1tPP/00I0eOJCYmhp49e3L48OFzr+QB4oluj7dKTEw0NVXGuUbkppI+92bC035gIYOIGP0Kl3eqJzceKFVF27Zto0OHDlVbyYtPeXlKef+fRGSdMSbxXOtqteFaqnjP9+S/ewvBRVlMCX2QUbf9lZgmDawOS6m6pR4lEnfQhFLbGEPW11MJXvkMxxwRfNZxJndfPxx/Xz3FpZSyliaU2qQwh9R37iRy/2d8ZRIpGv4qdye2tzoqpZQCNKHUGvYj28iacwPhefuYFTSWAbf9ndaRoVaHpZRSJ2hCqQWyf3wP/88ewO7wZ0aryfzxD7cQ5O9z7hWVUqoGaULxZiVFpCz4C822v8U6cxEHr3yNu/v1sDoqpeqNcZ+PA+rYY8A9SK/keilHxn5Spg6k2fa3WOA3nAZ3fsZwTSZK1Wql5es3bNhAnz596NSpE126dCEpKemMtu4oXw+wfv16Pv/8c7fEfy7aQ/FCuVuXYxbeTkN7ITOjn+aGP96rj+tVqg5p0KABb731Fm3btuXQoUP06NGDwYMH07hx4xNtKlu+/lzWr1/P5s2bGTJkiFtiPxvtoXgTh4PDHz9Hg/dHk2JvyLK+87n9Tw9oMlGqjmnXrh1t27YFoHnz5kRGRpKWllbp9Xfs2MHgwYPp0aMHl156Kb/++isA8+fPp3PnziQkJDBgwADy8/N59tlnmTdvXrV6N1WlPRQvYfIyOPDmLbQ8uopltv5E3TSd69rEWB2WUnXSpB8n8cuxX87ZrrRN6bWUs2kf1p5Hej5yznan+/HHHykqKqJNmzaVXmf8+PHMnDmTNm3asHr1au69916WLVvGM888wzfffENUVBSZmZkEBQXx5JNPsnnzZiZPnlzl2KpKE4oXyPttHXlv/4Go4jTeDr+XYbc9SZOG+swwpeq6w4cPc8sttzB37lxstsqdMMrMzGTNmjWMGjXqxGclJSUA9O3bl1tvvZXRo0dz3XXXeSTms9GEYrHDX88g/NvHKTIhLOkxi5uGjdDnvSvlYZXtSXjyLq/s7GyuueYa/v73v9O7d+9Kr2eMISIiotxrKm+88QY//PADS5cuJSEhgeTkZHeGfE56DcUqxfnsmfVHor99mPXSkUM3fMH1w0dqMlGqHigqKmLkyJEnehNV0aRJE6Kjo088wtfhcLBx40YAdu/eTe/evXnuuedo0qQJBw8eJCQkhJycHLcfQ3k0oVigMHUnB1/uT6v9i1nU8A+0fuAzEjuW/4Q1pVTd8/7777Ny5UrmzJlz4nbgqtzFNX/+fKZPn05CQgKdOnVi6dKlADz44IPEx8cTHx/PoEGD6Ny5MwMHDmTjxo1069bN4xfltXx9DTvy0wcEf3IvJQZWdHiWa0ffps8uUaoGVKd8fX0c2Kjl62sDewk733+MC7fPYAutyf7dLEYldrc6KqXUWdSnROIOmlBqQFFmCgdn/p4Lc9fzReDVdL7jNTpFNLE6LKWUcitNKB6WtvVbbAvHEW3P5sO4iQy95SF9dolSqk7ShOIpxrDj45eIW/8ih00EWwe9z4j+A62OSimlPEYTigeU5GWxc9ZttE//kv/59qL5uDn0b9Hc6rCUUsqjNKG4WfrejRS8cxNtiw/wabO7GHj73wn011pcStVGv91yKwAXvP2WxZHUDnoy341+/XI2QXOuIqA4m+8umcXQuydpMlFKnVDT5esXL17MSy+95Lb4z0V7KG7gKC5k85z76HIwiU22DgT9YS6XXXiR1WEppbyUO8vXl5SU4Otb/lf5yJEj3R/8WWhCOU+ZKXs5+uaNdCnaxleNr6fn+Gk0bBBkdVhKKS/Wrt3Jyhhly9eXTShn069fPy677DJWrVrFddddR6tWrfjHP/5BUVERTZs25Z133iEyMpKZM2eeqDR88803Ex4ezk8//URKSgovv/yy2xOOJpTzsHPNUsI/v5tmpohvu77EgBF3IqK1uJTydin/+AeF285dvr7gF2eb0mspZxPQoT3NHn+8yrFUp3w9OItLrly5EoCMjAyGDx+OiDB9+nRefvllJk2adMY6qamprF69mk2bNjFmzBhNKN7AOOz8PO8Juu78L7/ZYigYNYfL4s9ZlUAppU5RnfL1pW688cYT0/v27WPMmDGkpKRQWFh4Sg+orBEjRiAidOnShYMHD55X7OXRhFJFOZlp7J1xM93z1rCm4QDa3/kmjRuHWR2WUqoKKtuT8ORdXtUtX18qODj4xPQ999zD448/ztChQ/nyyy958cUXy10nIODkc5Y8UcdRE0oV7EleTcDicVzkOMrKdo/Q78ZHsWlhR6VUFZ1P+fryZGVl0aJFC4wxzJ071w0RVo9+G1bSTx9Mpvmia/ExJewY+j6X3vS4JhOlVLWcb/n60z399NOMHDmSyy67jKioKDdGWjVavv4cjMPBj9PG0uvYEjYFdCP6tnlERLXwUIRKKU+pTvn6+jiwUcvXe5DYbJiwNvyvwW30/ONL+FRwv7dSqu6pT4nEHfTbsRJ63/y01SEopZTXs+QigIiEichyEdnhej/j4SAiMkBENpR5FYjICNeyViLyg2v9JBHxr/mjUEopVZZVV5UfBVYYY9oCK1zzpzDGfG2M6WqM6QoMBPKAZa7Fk4BXXOtnALfXTNhKqdqsPl0zro7z/f9jVUK5Fii9t20uMOIc7a8HPjPG5IlzKPpAoLRCWmXWV0rVc4GBgaSnp2tSqYAxhvT0dAIDA6u9DauuoUQZYw4DGGMOi0jkOdrfCPzbNR0OZBpjSlzzBwC97UopdVYxMTEcOHCAtLQ0q0PxWoGBgcTExFR7fY8lFBH5EmhWzqKJVdxONBAPfFH6UTnNKvyTQ0TGA+MBYmNjq7JrpVQd4ufnR6tWrawOo07zWEIxxgyqaJmIHBGRaFfvJBpIPcumxgCLjTHFrvmjQGMR8XX1UmKAQ2eJYwYwA5zjUKp6HEoppSrHqmsoS4CxrumxwEdnaft74L3SGeM8Afo1zusqlVlfKaVUDbAqobwIXCkiO4ArXfOISKKIzCxtJCJxQEvg29PWfwT4s4jsxHlNZVYNxKyUUuos6lXpFRFJA36r5uoROE+31QV15VjqynGAHou3qivHcr7HcYExpum5GtWrhHI+RGRtZWrZ1AZ15VjqynGAHou3qivHUlPHoeVylVJKuYUmFKWUUm6hCaXyZlgdgBvVlWOpK8cBeizeqq4cS40ch15DUUop5RbaQ1FKKeUWmlCqQESeE5FkVzn9ZSLS3OqYqktEXhKRX1zHs1hEGlsdU3WIyGgR2SIiDhGplXfjiMgQEdkuIjtF5IzK27WFiLwpIqkistnqWM6HiLQUka9FZJvrZ2uC1TFVl4gEisiPIrLRdSzPeHR/esqr8kQk1BiT7Zq+H+hojLnL4rCqRUSuAr4yxpSIyCQAY8wjFodVZSLSAXAArwMPGWOq9oxni4mID/ArzgG+B4CfgN8bY7ZaGlg1iMilQC7wljGms9XxVJerHFS0MWa9iIQA64ARtfTfRIBgY0yuiPgB3wETjDFrPLE/7aFUQWkycQnmLEUpvZ0xZlmZis1rcNZEq3WMMduMMdutjuM89AR2GmN2G2OKgPk4H+9Q6xhjVgLHrI7jfBljDhtj1rumc4Bt1NKK5sYp1zXr53p57HtLE0oVicjzIrIfuAl40up43OQ24DOrg6inWgD7y8zr4xi8iKv8UzfgB2sjqT4R8RGRDTiL8C43xnjsWDShnEZEvhSRzeW8rgUwxkw0xrQE5gH3Whvt2Z3rWFxtJgIlOI/HK1XmOGqxKj2OQdUcEWkILAIeOO3sRK1ijLG7nnwbA/QUEY+djrTqAVte62xl90/zLvAJ8JQHwzkv5zoWERkLDAOuMF58Ma0K/ya10QGcBVBLnfVxDKpmuK43LALmGWM+sDoedzDGZIrIN8AQwCM3TmgPpQpEpG2Z2eHAL1bFcr5EZAjOqs3DjTF5VsdTj/0EtBWRViLij/PppEssjqlec13IngVsM8b8+1ztvZmINC29g1NEgoBBePB7S+/yqgIRWQRchPOuot+Au4wxB62Nqnpcpf8DgHTXR2tq4x1rIjIS+A/QFMgENhhjBlsbVdWIyFBgMuADvGmMed7ikKpFRN4DLsdZ2fYI8JQxptY9WkJE+gGrgE04f9cBHjfGfGpdVNUjIl2AuTh/tmzA+8aYZz22P00oSiml3EFPeSmllHILTShKKaXcQhOKUkopt9CEopRSyi00oSillHILTShKuZGI5J671VnXXygirV3TDUXkdRHZ5aoUu1JEeomIv2taByYrr6IJRSkvISKdAB9jzG7XRzNxFltsa4zpBPwRiHAVkVwB3GBJoEpVQBOKUh4gTi+5ao5tEpEbXJ/bROS/rh7HUhH5VESud612E/CRq10boBfwN2OMA8BVkfgTV9sPXe2V8hraZVbKM64DugIJOEeO/yQiK4G+QBwQD0TiLI3+pmudvsB7rulOOEf92yvY/mbgYo9ErlQ1aQ9FKc/oB7znqvR6BPgWZwLoBywwxjiMMSnA12XWiQbSKrNxV6Ipcj0ASimvoAlFKc8oryz92T4HyAcCXdNbgAQROdvvaABQUI3YlPIITShKecZK4AbXw42aApcCP+J8BOso17WUKJzFFEttAy4EMMbsAtYCz7iq3yIibUufASMi4UCaMaa4pg5IqXPRhKKUZywGkoGNwFfAX12nuBbhfAbKZuB1nE8CzHKt8wmnJpg7gGbAThHZBLzByWelDABqXfVbVbdptWGlapiINDTG5Lp6GT8CfY0xKa7nVXztmq/oYnzpNj4AHjPGbK+BkJWqFL3LS6mat9T10CN/4DlXzwVjTL6IPIXzmfL7KlrZ9SCuDzWZKG+jPRSllFJuoddQlFJKuYUmFKWUUm6hCUUppZRbaEJRSinlFppQlFJKuYUmFKWUUm7x/wBqIuAhxRQlAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot CV误差曲线\n",
    "test_means = grid.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = grid.cv_results_[ 'std_test_score' ]\n",
    "train_means = grid.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = grid.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "# plot results\n",
    "n_Cs = len(Cs)\n",
    "number_penaltys = len(penaltys)\n",
    "test_scores = np.array(test_means).reshape(n_Cs,number_penaltys)\n",
    "train_scores = np.array(train_means).reshape(n_Cs,number_penaltys)\n",
    "test_stds = np.array(test_stds).reshape(n_Cs,number_penaltys)\n",
    "train_stds = np.array(train_stds).reshape(n_Cs,number_penaltys)\n",
    "\n",
    "x_axis = np.log10(Cs)\n",
    "for i, value in enumerate(penaltys):\n",
    "    #pyplot.plot(log(Cs), test_scores[i], label= 'penalty:'   + str(value))\n",
    "    pyplot.errorbar(x_axis, test_scores[:,i], yerr=test_stds[:,i] ,label = penaltys[i] +' Test')\n",
    "    pyplot.errorbar(x_axis, train_scores[:,i], yerr=train_stds[:,i] ,label = penaltys[i] +' Train')\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'logloss' )\n",
    "pyplot.savefig('LogisticGridSearchCV_C.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 用LogisticRegressionCV实现正则化的 Logistic Regression"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L1正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l1',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1,10,100, 1000]\n",
    "\n",
    "lrcv_L1 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l1', solver='liblinear', multi_class='ovr')\n",
    "\n",
    "lrcv_L1.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.49423702, -0.49550376, -0.49564707, -0.49566646],\n",
       "        [-0.50930409, -0.50874893, -0.50870717, -0.50870517],\n",
       "        [-0.48179891, -0.48313625, -0.4832771 , -0.48329262],\n",
       "        [-0.43661994, -0.43568774, -0.43560591, -0.43559907],\n",
       "        [-0.48810242, -0.48809015, -0.48810248, -0.48810083]])}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C is: [1]\n"
     ]
    }
   ],
   "source": [
    "print ('C is:',lrcv_L1.C_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.40471897,  1.10208211, -0.23905837,  0.        , -0.12013776,\n",
       "         0.68832342,  0.30156618,  0.16761985]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lrcv_L1.coef_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### L2正则"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LogisticRegressionCV(Cs=[1, 10, 100, 1000], class_weight=None, cv=5,\n",
       "           dual=False, fit_intercept=True, intercept_scaling=1.0,\n",
       "           max_iter=100, multi_class='ovr', n_jobs=1, penalty='l2',\n",
       "           random_state=None, refit=True, scoring='neg_log_loss',\n",
       "           solver='liblinear', tol=0.0001, verbose=0)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "\n",
    "Cs = [1, 10,100,1000]\n",
    "\n",
    "# 大量样本（6W+）、高维度（93），L2正则 --> 缺省用lbfgs，为了和GridSeachCV比较，也用liblinear\n",
    "\n",
    "lr_cv_L2 = LogisticRegressionCV(Cs=Cs, cv = 5, scoring='neg_log_loss', penalty='l2', solver='liblinear', multi_class='ovr')\n",
    "lr_cv_L2.fit(X_train, y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1: array([[-0.49442383, -0.49553449, -0.49565533, -0.49566752],\n",
       "        [-0.50834194, -0.50865995, -0.5086969 , -0.50870065],\n",
       "        [-0.4830274 , -0.48326216, -0.48329051, -0.4832934 ],\n",
       "        [-0.43615578, -0.43565059, -0.43560284, -0.43559809],\n",
       "        [-0.48793522, -0.48807935, -0.48809877, -0.48810077]])}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.scores_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C is: [1]\n"
     ]
    }
   ],
   "source": [
    "print ('C is:',lr_cv_L2.C_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.4079442 ,  1.10555527, -0.25049493,  0.00913944, -0.13088899,\n",
       "         0.69442933,  0.30860251,  0.17576207]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr_cv_L2.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
